{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.8.1-final"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3,
  "kernelspec": {
   "name": "python38164bitc9425b4fd2d44566b8cda82b00a1f4f8",
   "display_name": "Python 3.8.1 64-bit"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np \n",
    "import scipy.spatial.distance as ssd \n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>rating</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>4</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>5</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>79995</th>\n      <td>943</td>\n      <td>1067</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>79996</th>\n      <td>943</td>\n      <td>1074</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>79997</th>\n      <td>943</td>\n      <td>1188</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>79998</th>\n      <td>943</td>\n      <td>1228</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>79999</th>\n      <td>943</td>\n      <td>1330</td>\n      <td>3</td>\n    </tr>\n  </tbody>\n</table>\n<p>80000 rows × 3 columns</p>\n</div>",
      "text/plain": "       user_id  item_id  rating\n0            1        1       5\n1            1        2       3\n2            1        3       4\n3            1        4       3\n4            1        5       3\n...        ...      ...     ...\n79995      943     1067       2\n79996      943     1074       4\n79997      943     1188       3\n79998      943     1228       3\n79999      943     1330       3\n\n[80000 rows x 3 columns]"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\n",
    "    './movielen_rating_training.base',\n",
    "    names = ['user_id', 'item_id', 'rating'],\n",
    "    usecols = [0,1,2],\n",
    "    sep = '\\t'\n",
    ")\n",
    "df_test = pd.read_csv(\n",
    "    './movielen_rating_test.base',\n",
    "    names = ['user_id', 'item_id', 'rating'],\n",
    "    usecols = [0,1,2],\n",
    "    sep = '\\t'\n",
    ")\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([  1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,\n        14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,\n        27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,\n        40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,  52,\n        53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,  65,\n        66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,\n        79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  91,\n        92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103, 104,\n       105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,\n       118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130,\n       131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,\n       144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156,\n       157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,\n       170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182,\n       183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195,\n       196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208,\n       209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221,\n       222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234,\n       235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247,\n       248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260,\n       261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273,\n       274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286,\n       287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299,\n       300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312,\n       313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325,\n       326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338,\n       339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351,\n       352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364,\n       365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377,\n       378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390,\n       391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403,\n       404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416,\n       417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429,\n       430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442,\n       443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455,\n       456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468,\n       469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481,\n       482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494,\n       495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507,\n       508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520,\n       521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533,\n       534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546,\n       547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559,\n       560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572,\n       573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585,\n       586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598,\n       599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611,\n       612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624,\n       625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637,\n       638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650,\n       651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663,\n       664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676,\n       677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689,\n       690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702,\n       703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715,\n       716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728,\n       729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741,\n       742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754,\n       755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767,\n       768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780,\n       781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793,\n       794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806,\n       807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819,\n       820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832,\n       833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845,\n       846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858,\n       859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871,\n       872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884,\n       885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897,\n       898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910,\n       911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923,\n       924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936,\n       937, 938, 939, 940, 941, 942, 943])"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.user_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "uid2index = dict(\n",
    "    (uid,index) \n",
    "    for (index, uid) in enumerate(df_train.user_id.unique()))\n",
    "item2index = dict(\n",
    "    (item_id,index) \n",
    "    for (index, item_id) in enumerate(df_train.item_id.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>rating</th>\n      <th>user_index</th>\n      <th>item_index</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1</td>\n      <td>5</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>3</td>\n      <td>4</td>\n      <td>0</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>4</td>\n      <td>3</td>\n      <td>0</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>5</td>\n      <td>3</td>\n      <td>0</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>79995</th>\n      <td>943</td>\n      <td>1067</td>\n      <td>2</td>\n      <td>942</td>\n      <td>901</td>\n    </tr>\n    <tr>\n      <th>79996</th>\n      <td>943</td>\n      <td>1074</td>\n      <td>4</td>\n      <td>942</td>\n      <td>906</td>\n    </tr>\n    <tr>\n      <th>79997</th>\n      <td>943</td>\n      <td>1188</td>\n      <td>3</td>\n      <td>942</td>\n      <td>1027</td>\n    </tr>\n    <tr>\n      <th>79998</th>\n      <td>943</td>\n      <td>1228</td>\n      <td>3</td>\n      <td>942</td>\n      <td>1082</td>\n    </tr>\n    <tr>\n      <th>79999</th>\n      <td>943</td>\n      <td>1330</td>\n      <td>3</td>\n      <td>942</td>\n      <td>1291</td>\n    </tr>\n  </tbody>\n</table>\n<p>80000 rows × 5 columns</p>\n</div>",
      "text/plain": "       user_id  item_id  rating  user_index  item_index\n0            1        1       5           0           0\n1            1        2       3           0           1\n2            1        3       4           0           2\n3            1        4       3           0           3\n4            1        5       3           0           4\n...        ...      ...     ...         ...         ...\n79995      943     1067       2         942         901\n79996      943     1074       4         942         906\n79997      943     1188       3         942        1027\n79998      943     1228       3         942        1082\n79999      943     1330       3         942        1291\n\n[80000 rows x 5 columns]"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['user_index'] = df_train.user_id.apply(lambda i: uid2index[i])\n",
    "df_train['item_index'] = df_train.item_id.apply(lambda i: item2index[i])\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(943, 1650)"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_count = len(df_train.user_id.unique())\n",
    "item_count = len(df_train.item_id.unique())\n",
    "(user_count, item_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([3.68148148, 3.8       , 3.        , 4.35714286, 2.95604396,\n       3.58181818, 3.89201878, 3.6       , 4.16666667, 4.21276596,\n       3.53333333, 4.28      , 3.13672922, 4.2195122 , 3.03333333,\n       4.34782609, 3.15789474, 3.93710692, 3.6       , 3.30769231,\n       2.66315789, 3.3       , 3.63636364, 4.3902439 , 4.04878049,\n       2.90909091, 3.3       , 3.64102564, 3.94117647, 3.8       ,\n       3.91304348, 3.54545455, 3.64285714, 3.8       , 3.        ,\n       4.        , 3.45945946, 3.9047619 , 3.84615385, 2.72727273,\n       3.93548387, 3.625     , 3.67857143, 3.6375    , 3.48275862,\n       4.18181818, 3.5625    , 3.72727273, 2.72897196, 3.53846154,\n       3.75      , 4.34285714, 4.        , 3.33333333, 3.8       ,\n       3.65217391, 3.62962963, 3.875     , 4.02325581, 4.13445378,\n       2.83333333, 3.31707317, 2.97916667, 3.59633028, 3.97916667,\n       3.52380952, 3.41666667, 3.16666667, 3.73684211, 3.4025974 ,\n       3.81818182, 3.76712329, 3.64705882, 3.76190476, 3.18181818,\n       3.4       , 3.46153846, 3.35714286, 4.18518519, 3.92857143,\n       3.56666667, 3.12790698, 3.3974359 , 3.74418605, 3.54716981,\n       3.63636364, 3.77876106, 4.09090909, 4.15      , 4.25      ,\n       3.9       , 3.23039216, 3.        , 3.66063348, 3.48407643,\n       4.25      , 3.97142857, 3.875     , 3.65079365, 3.02941176,\n       3.        , 2.5625    , 3.4       , 2.79365079, 3.3125    ,\n       3.71428571, 2.76923077, 3.41176471, 3.5       , 3.08536585,\n       3.5       , 3.91666667, 3.80645161, 3.62962963, 4.04081633,\n       3.05681818, 3.72222222, 4.64285714, 3.87755102, 3.57142857,\n       3.67391304, 3.97058824, 3.93103448, 3.5       , 3.44444444,\n       3.69565217, 4.18181818, 3.57843137, 2.86666667, 4.07865169,\n       4.06666667, 3.70588235, 3.27777778, 3.58333333, 3.45      ,\n       4.55      , 4.08333333, 4.2962963 , 3.93333333, 3.6       ,\n       3.44615385, 3.8       , 3.73333333, 3.66666667, 3.31891892,\n       3.68421053, 4.3       , 4.        , 3.        , 3.8       ,\n       4.08433735, 4.28571429, 3.07692308, 3.96296296, 2.63636364,\n       3.77777778, 3.7037037 , 3.83838384, 3.4057971 , 4.09090909,\n       2.52941176, 3.35      , 3.07692308, 4.06060606, 4.        ,\n       3.3       , 3.81578947, 3.25714286, 3.95652174, 3.66666667,\n       3.42105263, 3.        , 4.18518519, 3.80208333, 3.82608696,\n       3.64285714, 3.73015873, 3.71895425, 2.91304348, 3.85294118,\n       1.51834862, 4.14285714, 3.16666667, 3.71428571, 3.875     ,\n       3.33333333, 4.20588235, 3.89655172, 3.96330275, 3.38888889,\n       4.        , 3.45      , 3.27868852, 2.94444444, 3.2244898 ,\n       3.23809524, 3.19230769, 3.25471698, 3.15789474, 4.        ,\n       3.14418605, 2.84615385, 3.27272727, 3.63157895, 2.8       ,\n       1.87878788, 3.21804511, 3.72727273, 3.375     , 3.98571429,\n       3.38095238, 4.23529412, 4.23880597, 3.87301587, 3.86      ,\n       3.93150685, 3.05405405, 3.60606061, 4.125     , 3.8       ,\n       3.61445783, 3.03286385, 3.19230769, 3.09375   , 4.8       ,\n       3.64516129, 3.27777778, 2.85714286, 3.13333333, 3.92      ,\n       3.6       , 3.89361702, 4.35294118, 3.10431655, 3.89361702,\n       3.35714286, 4.11111111, 3.21428571, 4.11627907, 3.6       ,\n       3.33333333, 4.27272727, 3.55555556, 3.70940171, 3.30769231,\n       2.95495495, 3.8       , 3.19047619, 4.19101124, 3.83076923,\n       3.72727273, 4.54545455, 3.94642857, 3.15294118, 2.47826087,\n       4.06956522, 4.25806452, 3.625     , 3.81481481, 4.16666667,\n       4.1       , 3.20731707, 4.05      , 4.19047619, 3.62962963,\n       3.1875    , 3.99056604, 2.93296089, 2.8502994 , 4.29166667,\n       3.59006211, 4.38709677, 3.5       , 4.02564103, 3.0625    ,\n       3.41608392, 3.28571429, 4.16666667, 3.17355372, 3.65306122,\n       3.29411765, 3.4       , 4.07142857, 3.60714286, 3.8125    ,\n       3.83544304, 3.85714286, 3.72093023, 2.61538462, 3.41772152,\n       3.69364162, 4.14285714, 3.09767442, 3.5       , 4.29      ,\n       4.25675676, 3.40860215, 4.19230769, 3.50649351, 3.75      ,\n       3.58940397, 2.5       , 3.36594203, 3.46153846, 3.424     ,\n       3.33333333, 3.66666667, 3.83486239, 3.81818182, 3.        ,\n       3.75675676, 4.48461538, 3.64705882, 3.67142857, 3.97777778,\n       3.11363636, 3.58333333, 3.6875    , 3.25      , 3.73417722,\n       3.81944444, 4.22222222, 3.63636364, 4.39215686, 3.35526316,\n       3.22123894, 3.28      , 3.40828402, 3.2972973 , 4.44705882,\n       3.65      , 4.13793103, 3.78571429, 3.53211009, 3.61538462,\n       3.01176471, 4.04761905, 4.12195122, 4.06329114, 4.11111111,\n       3.66666667, 3.4765625 , 3.98726115, 3.69724771, 3.6641791 ,\n       3.32835821, 3.60330579, 3.77419355, 3.35714286, 4.4       ,\n       4.13333333, 3.79310345, 3.28571429, 3.71710526, 4.1       ,\n       3.46666667, 4.05769231, 3.71428571, 4.14285714, 3.97101449,\n       3.75641026, 3.27777778, 3.0861244 , 3.35714286, 3.36842105,\n       4.35294118, 4.11428571, 3.3125    , 4.        , 3.67857143,\n       4.1025641 , 4.22727273, 3.71794872, 3.31958763, 3.9375    ,\n       3.63636364, 4.        , 3.40530303, 4.01470588, 3.1637931 ,\n       3.79775281, 3.54545455, 4.42307692, 4.1875    , 3.30693069,\n       3.83333333, 3.33333333, 4.11904762, 3.70792079, 3.75      ,\n       3.75789474, 4.07407407, 3.33968254, 3.808     , 3.82608696,\n       3.30769231, 3.8375    , 3.62307692, 2.92      , 3.64705882,\n       3.11504425, 3.83333333, 3.63829787, 3.25      , 1.84879725,\n       3.47445255, 3.4787234 , 3.7826087 , 3.70322581, 3.08695652,\n       3.74      , 3.94594595, 3.675     , 3.95454545, 4.28571429,\n       3.85131894, 3.23432343, 2.86666667, 4.17241379, 4.21875   ,\n       3.81818182, 3.4125    , 3.52941176, 3.3       , 2.94382022,\n       3.81609195, 4.58333333, 3.9375    , 3.37087912, 3.50819672,\n       3.27777778, 3.66666667, 3.45945946, 3.73170732, 3.30835735,\n       3.74603175, 3.57142857, 4.06896552, 3.71428571, 4.14583333,\n       3.5       , 3.14516129, 3.375     , 3.71428571, 2.00819672,\n       2.74285714, 3.57480315, 3.37142857, 3.74626866, 3.8582996 ,\n       2.73626374, 3.43877551, 3.27516779, 2.97333333, 3.5       ,\n       3.46948357, 4.0260223 , 3.68333333, 3.38461538, 3.41791045,\n       3.        , 3.96875   , 2.86466165, 4.03773585, 3.4375    ,\n       3.46534653, 3.68181818, 3.99300699, 4.53488372, 3.57894737,\n       3.38709677, 4.22813688, 3.91428571, 4.08256881, 3.6       ,\n       3.37349398, 4.45714286, 3.4159292 , 3.42079208, 3.73333333,\n       3.98214286, 3.42307692, 3.05084746, 3.99280576, 3.03846154,\n       3.22222222, 3.64502165, 3.35714286, 3.72477064, 2.78947368,\n       3.84848485, 3.52727273, 3.75555556, 3.87234043, 3.95305164,\n       3.03100775, 3.17921147, 3.2885906 , 3.8627451 , 3.34222222,\n       3.76712329, 3.10810811, 4.01875   , 3.62151394, 3.36607143,\n       3.67355372, 4.72413793, 3.82758621, 2.51515152, 2.76666667,\n       3.75      , 4.33333333, 4.36363636, 3.84020619, 3.        ,\n       4.0952381 , 3.2972973 , 3.64383562, 4.12244898, 3.43478261,\n       3.14      , 4.33333333, 4.36      , 3.49346405, 3.2962963 ,\n       3.23529412, 3.83211679, 3.71698113, 3.60465116, 3.88888889,\n       3.23333333, 4.12043796, 3.38461538, 4.075     , 3.93577982,\n       3.87116564, 2.86530612, 3.61728395, 3.98214286, 3.71428571,\n       3.62406015, 3.48888889, 3.53299492, 2.80645161, 3.50617284,\n       3.89830508, 3.65217391, 3.67096774, 3.72      , 3.675     ,\n       3.80239521, 3.0952381 , 4.17      , 3.57017544, 4.01923077,\n       4.20454545, 3.75471698, 4.2       , 3.57894737, 3.3960396 ,\n       3.02521008, 3.54166667, 3.76666667, 3.41176471, 4.54285714,\n       3.44295302, 3.95483871, 3.37037037, 3.48571429, 2.68181818,\n       3.45      , 3.25      , 3.74509804, 3.52173913, 3.15384615,\n       3.52777778, 3.80319149, 2.66666667, 3.47297297, 3.55319149,\n       4.03571429, 3.27419355, 4.44444444, 3.54166667, 3.7375    ,\n       3.3253012 , 2.96938776, 3.74222222, 3.44186047, 3.5       ,\n       3.6547619 , 3.81388889, 3.60897436, 3.48      , 3.19791667,\n       3.6       , 3.63414634, 3.68      , 3.89361702, 3.58426966,\n       3.08108108, 3.79310345, 3.78723404, 3.2962963 , 3.75555556,\n       3.96610169, 3.83783784, 3.67484663, 2.39285714, 3.73333333,\n       3.83333333, 3.37037037, 4.32142857, 3.12820513, 3.86407767,\n       3.62790698, 2.62037037, 3.28703704, 3.50561798, 3.82727273,\n       3.5497076 , 3.39035088, 3.73333333, 3.53191489, 3.43396226,\n       2.34375   , 3.29281768, 4.7037037 , 4.08264463, 3.32407407,\n       3.1       , 3.66101695, 3.31034483, 3.37037037, 3.26470588,\n       4.45      , 2.49038462, 3.53731343, 2.95945946, 4.2293578 ,\n       4.125     , 3.62264151, 3.73786408, 4.        , 3.95901639,\n       2.94736842, 3.82758621, 3.25925926, 3.79166667, 3.15434084,\n       3.28571429, 3.13636364, 2.69257951, 3.68707483, 2.9080292 ,\n       2.54166667, 3.24137931, 3.73239437, 3.79057592, 2.58482143,\n       3.91735537, 3.86956522, 3.53164557, 3.74698795, 3.47887324,\n       3.66530612, 3.95744681, 3.52173913, 3.38      , 3.84782609,\n       3.58064516, 3.24137931, 3.88571429, 3.65853659, 3.70588235,\n       3.58441558, 3.59183673, 3.42857143, 3.66129032, 4.        ,\n       3.22727273, 3.13784461, 3.22666667, 3.48837209, 2.05      ,\n       4.56338028, 3.47619048, 4.83333333, 3.83333333, 3.14782609,\n       4.21875   , 3.275     , 3.13636364, 4.15286624, 3.28947368,\n       3.93103448, 3.71428571, 2.84251969, 3.14765101, 3.66666667,\n       4.21212121, 2.4516129 , 3.53191489, 3.77777778, 3.71052632,\n       3.31034483, 3.49576271, 3.2       , 3.57246377, 3.69767442,\n       3.85972851, 3.7654321 , 2.96666667, 3.52272727, 3.38323353,\n       3.88847584, 3.70967742, 3.69230769, 3.10447761, 3.96666667,\n       3.53571429, 3.61904762, 3.27272727, 2.16470588, 3.8       ,\n       3.        , 3.03726708, 3.57692308, 2.85714286, 3.23684211,\n       3.38554217, 3.7       , 3.02803738, 3.609375  , 3.32075472,\n       3.        , 3.96969697, 3.67567568, 3.6       , 3.4       ,\n       3.46153846, 3.46153846, 3.68571429, 3.85185185, 3.1       ,\n       3.64179104, 4.05460751, 3.72222222, 3.62295082, 3.09090909,\n       3.54166667, 3.20547945, 3.59649123, 3.38235294, 3.10526316,\n       3.19266055, 3.36746988, 3.8767507 , 4.0625    , 3.24390244,\n       2.84848485, 3.04761905, 3.78294574, 3.68807339, 4.08695652,\n       3.35428571, 4.43243243, 3.265625  , 3.06896552, 4.296875  ,\n       3.71428571, 4.        , 3.27950311, 2.05803571, 3.78571429,\n       3.46666667, 4.05555556, 2.95384615, 3.91891892, 3.87272727,\n       3.78571429, 2.7887931 , 3.86666667, 3.84615385, 3.7037037 ,\n       3.87179487, 3.14035088, 3.34136546, 3.66666667, 3.01731602,\n       3.8       , 3.12765957, 3.50909091, 4.30769231, 3.13207547,\n       3.60055866, 2.65384615, 3.32635983, 4.        , 3.75      ,\n       4.03846154, 3.49333333, 3.09090909, 3.66566265, 3.34798535,\n       3.64285714, 3.89215686, 4.13043478, 3.45      , 4.42307692,\n       4.14285714, 3.5       , 2.79310345, 2.97142857, 3.7431694 ,\n       3.84      , 3.36111111, 3.47619048, 4.07142857, 3.        ,\n       4.33870968, 3.12      , 4.00540541, 2.6       , 3.6137931 ,\n       3.82608696, 3.25      , 3.11538462, 3.546875  , 3.74528302,\n       3.52054795, 2.96      , 3.05617978, 3.92592593, 4.14705882,\n       3.86      , 3.04347826, 4.11827957, 3.19298246, 4.13705584,\n       3.90322581, 3.74074074, 2.73529412, 3.50617284, 3.12903226,\n       3.74074074, 3.03424658, 4.28666667, 4.86956522, 4.52941176,\n       3.45933014, 3.46938776, 2.97560976, 3.21198157, 3.34782609,\n       3.42307692, 3.27272727, 3.42857143, 3.84615385, 3.23943662,\n       4.        , 4.36363636, 3.12149533, 3.82312925, 2.28787879,\n       2.7       , 4.26315789, 2.95192308, 2.87234043, 3.4535316 ,\n       3.49565217, 3.38028169, 2.9       , 3.82352941, 4.19101124,\n       4.19047619, 3.82716049, 3.26865672, 3.68965517, 3.42663043,\n       3.32046332, 4.05109489, 3.92537313, 3.72093023, 3.31372549,\n       3.3625    , 3.92397661, 4.3       , 3.43558282, 3.85714286,\n       4.12765957, 4.        , 3.42372881, 3.63673469, 3.75      ,\n       2.98066298, 3.96216216, 3.5       , 3.52592593, 2.55555556,\n       3.85483871, 3.44897959, 3.85294118, 3.74468085, 3.275     ,\n       3.80487805, 4.57142857, 3.55405405, 4.34615385, 3.16666667,\n       3.80612245, 3.81132075, 3.52272727, 3.08695652, 3.11538462,\n       3.3659306 , 3.54285714, 3.34951456, 3.47004608, 3.23076923,\n       3.27272727, 3.37007874, 4.14864865, 3.75609756, 3.125     ,\n       3.3       , 3.69166667, 4.6875    , 3.69387755, 2.96825397,\n       3.72131148, 3.96680498, 2.64673913, 3.70114943, 3.92307692,\n       3.74647887, 3.375     , 3.26851852, 4.26530612, 3.45794393,\n       4.04545455, 4.26582278, 3.41071429])"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_rating_mean = np.array([\n",
    "    grouped.rating.mean() \n",
    "    for uix, grouped in df_train.groupby('user_index')\n",
    "])\n",
    "user_rating_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([[5., 4., 0., ..., 5., 0., 0.],\n       [3., 0., 0., ..., 0., 0., 5.],\n       [4., 0., 0., ..., 0., 0., 0.],\n       ...,\n       [0., 0., 0., ..., 0., 0., 0.],\n       [0., 0., 0., ..., 0., 0., 0.],\n       [0., 0., 0., ..., 0., 0., 0.]])"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_rating_matrix = np.zeros((item_count,user_count))\n",
    "for (uix, iix), grouped in df_train.groupby(['user_index', 'item_index']):\n",
    "    item_rating_matrix[iix, uix] = grouped.rating.mean()\n",
    "item_rating_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array([[ 1.31851852,  0.2       , -0.        , ...,  0.95454545,\n        -0.        , -0.        ],\n       [-0.68148148, -0.        , -0.        , ..., -0.        ,\n        -0.        ,  1.58928571],\n       [ 0.31851852, -0.        , -0.        , ..., -0.        ,\n        -0.        , -0.        ],\n       ...,\n       [-0.        , -0.        , -0.        , ..., -0.        ,\n        -0.        , -0.        ],\n       [-0.        , -0.        , -0.        , ..., -0.        ,\n        -0.        , -0.        ],\n       [-0.        , -0.        , -0.        , ..., -0.        ,\n        -0.        , -0.        ]])"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# assume rating != mean rating\n",
    "# more advanced can be achieved by np.mq\n",
    "item_rating_unbised_matrix = (\n",
    "        item_rating_matrix  - user_rating_mean.reshape(user_count)\n",
    "    ) * (item_rating_matrix > 0) \n",
    "item_rating_unbised_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(1650, 1650)"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "item_sim = pairwise_distances(item_rating_unbised_matrix, metric='cosine')\n",
    "item_sim.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(1650, 943)"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_pred = np.clip((np.dot(item_rating_matrix.T,item_sim) / np.sum(np.abs(item_sim),axis = 1)).T + user_rating_mean,1,5)\n",
    "item_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(943, 1650)"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_pred = item_pred.T\n",
    "item_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(943, 50)"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend = np.resize(np.argsort(item_pred, axis  = 1),(user_count, 50))\n",
    "recommend.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>item_id</th>\n      <th>rating</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>6</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>10</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>12</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>14</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>17</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>19995</th>\n      <td>458</td>\n      <td>648</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>19996</th>\n      <td>458</td>\n      <td>1101</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>19997</th>\n      <td>459</td>\n      <td>934</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>19998</th>\n      <td>460</td>\n      <td>10</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>19999</th>\n      <td>462</td>\n      <td>682</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n<p>19968 rows × 3 columns</p>\n</div>",
      "text/plain": "       user_id  item_id  rating\n0            1        6       5\n1            1       10       3\n2            1       12       5\n3            1       14       5\n4            1       17       3\n...        ...      ...     ...\n19995      458      648       4\n19996      458     1101       4\n19997      459      934       3\n19998      460       10       3\n19999      462      682       5\n\n[19968 rows x 3 columns]"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = df_test[df_test.user_id.isin(uid2index) & df_test.item_id.isin(item2index)]\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "recommend_set_list = [\n",
    "    set(recommend[i])\n",
    "    for i in range(user_count)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(0.01009544008483563, 0.029020851115717596)"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ac_count = 0\n",
    "for (_, uid, iid, rating) in df_test[df_test.rating>=3].itertuples():\n",
    "    ac_count += 1 if item2index[iid] in recommend_set_list[uid2index[uid]] else 0\n",
    "predict_count = user_count * 50 \n",
    "test_count = len(df_test[df_test.rating>=3])\n",
    "(ac_count/ predict_count, ac_count/test_count)"
   ]
  }
 ]
}